import { CrawledData } from '@weubi/shared';

export interface ProcessedData {
  title?: string;
  content?: string;
  url: string;
  metadata?: Record<string, any>;
  extractedAt: Date;
}

export class DataProcessor {
  /**
   * 初始化数据处理器
   */
  async initialize(): Promise<void> {
    console.log('数据处理器已初始化');
  }

  /**
   * 处理原始爬取数据
   */
  async processData(rawData: any, url: string): Promise<ProcessedData> {
    try {
      // 基础数据处理
      const processedData: ProcessedData = {
        url,
        extractedAt: new Date(),
        metadata: {}
      };

      // 如果是 HTML 内容
      if (typeof rawData === 'string') {
        processedData.content = this.cleanHtmlContent(rawData);
        processedData.title = this.extractTitle(rawData);
      } 
      // 如果是结构化数据
      else if (typeof rawData === 'object') {
        processedData.title = rawData.title || this.extractTitle(rawData.html || '');
        processedData.content = rawData.content || this.cleanHtmlContent(rawData.html || '');
        processedData.metadata = rawData.metadata || {};
      }

      return processedData;
    } catch (error) {
      console.error('数据处理失败:', error);
      throw new Error(`数据处理失败: ${error instanceof Error ? error.message : '未知错误'}`);
    }
  }

  /**
   * 清理 HTML 内容
   */
  private cleanHtmlContent(html: string): string {
    if (!html) return '';
    
    // 移除 HTML 标签
    let text = html.replace(/<[^>]*>/g, '');
    
    // 清理多余的空白字符
    text = text.replace(/\s+/g, ' ').trim();
    
    // 解码 HTML 实体
    text = text
      .replace(/&amp;/g, '&')
      .replace(/&lt;/g, '<')
      .replace(/&gt;/g, '>')
      .replace(/&quot;/g, '"')
      .replace(/&#39;/g, "'")
      .replace(/&nbsp;/g, ' ');
    
    return text;
  }

  /**
   * 提取标题
   */
  private extractTitle(html: string): string {
    if (!html) return '';
    
    // 尝试提取 title 标签
    const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i);
    if (titleMatch && titleMatch[1]) {
      return this.cleanHtmlContent(titleMatch[1]);
    }
    
    // 尝试提取 h1 标签
    const h1Match = html.match(/<h1[^>]*>([^<]*)<\/h1>/i);
    if (h1Match && h1Match[1]) {
      return this.cleanHtmlContent(h1Match[1]);
    }
    
    return '';
  }

  /**
   * 保存处理后的数据到数据库
   */
  async saveData(processedData: ProcessedData, taskId: string): Promise<CrawledData> {
    try {
      // 创建数据库实体
      const crawledData = new CrawledData();
      crawledData.url = processedData.url;
      crawledData.data = {
        title: processedData.title || '',
        content: processedData.content || '',
        ...processedData
      };
      crawledData.metadata = processedData.metadata || {};
      crawledData.taskId = taskId;
      crawledData.hash = CrawledData.generateHash(crawledData.data, crawledData.url);

      // 这里应该保存到数据库
      // 暂时返回模拟数据
      console.log('保存数据到数据库:', {
        url: crawledData.url,
        title: crawledData.data.title,
        contentLength: crawledData.data.content?.length || 0,
        taskId: crawledData.taskId
      });

      return crawledData;
    } catch (error) {
      console.error('保存数据失败:', error);
      throw new Error(`保存数据失败: ${error instanceof Error ? error.message : '未知错误'}`);
    }
  }

  /**
   * 验证数据质量
   */
  validateData(data: ProcessedData): boolean {
    // 基础验证
    if (!data.url) {
      console.warn('数据验证失败: 缺少 URL');
      return false;
    }

    if (!data.content && !data.title) {
      console.warn('数据验证失败: 内容和标题都为空');
      return false;
    }

    // 内容长度验证
    if (data.content && data.content.length < 10) {
      console.warn('数据验证失败: 内容过短');
      return false;
    }

    return true;
  }
}
